{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "data_structures",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jo-cho/advances_in_financial_machine_learning/blob/master/data_structures.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgeWXpyE4Rsa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "e1a9dbfe-2272-443e-8bc9-a8522f21ec73"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z2uWyoj56cvw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NtrdAQty6K6g",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "227eeae4-d698-4bc7-934e-6e5be7faa838"
      },
      "source": [
        "!pip install -q mlfinlab\n",
        "from mlfinlab import data_structures, features, filters, labeling, util"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K     |██▊                             | 10kB 28.0MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 20kB 7.0MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 30kB 9.8MB/s eta 0:00:01\r\u001b[K     |███████████                     | 40kB 6.2MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 51kB 7.5MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 61kB 8.7MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 71kB 9.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 81kB 11.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 92kB 12.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 102kB 9.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 112kB 9.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 122kB 9.9MB/s \n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KG-RtLcV6Jz1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "6d3636a7-c366-459b-9012-136ce877758a"
      },
      "source": [
        "# 데이터 (Chapter2)\n",
        "\n",
        "raw_dollar_bars = data_structures.get_dollar_bars('/content/drive/My Drive/Colab Notebooks/csv/clean_IVE_tickbidask2.csv', threshold=1000000)\n",
        "dollar_bars = raw_dollar_bars.set_index(pd.to_datetime(raw_dollar_bars.date_time))\n",
        "dollar_bars = dollar_bars.drop(columns='date_time')\n",
        "dollar_bars = dollar_bars.reset_index().drop_duplicates(subset='date_time', keep='last').set_index('date_time')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading data in batches:\n",
            "Batch number: 0\n",
            "Returning bars \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9fWlRHI90R9",
        "colab_type": "text"
      },
      "source": [
        "y"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BrFbGlIPPScB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# triple-barrier (Chapter3)\n",
        "\n",
        "close = dollar_bars.close\n",
        "pt_sl = [1,1]\n",
        "target = util.get_daily_vol(close)\n",
        "h = target.mean()\n",
        "t_events = filters.cusum_filter(close,h)\n",
        "vertical_barrier_times = labeling.add_vertical_barrier(t_events,close,num_days=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TZBO_j-F7NB6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# side (Chapter3)\n",
        "\n",
        "fast_window = 20\n",
        "slow_window = 50\n",
        "\n",
        "df = pd.DataFrame()\n",
        "df['fast_mavg'] = close.rolling(window=fast_window, min_periods=fast_window, center=False).mean()\n",
        "df['slow_mavg'] = close.rolling(window=slow_window, min_periods=slow_window, center=False).mean()\n",
        "df['side'] = np.nan\n",
        "\n",
        "long_signals = df['fast_mavg'] >= df['slow_mavg'] \n",
        "short_signals = df['fast_mavg'] < df['slow_mavg'] \n",
        "df.loc[long_signals, 'side'] = 1\n",
        "df.loc[short_signals, 'side'] = -1\n",
        "\n",
        "df['side'] = df['side'].shift(1)\n",
        "\n",
        "side = df['side']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NhmWRXCN8P_P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# bin without side\n",
        "events = labeling.get_events(close, t_events, pt_sl, target, min_ret=0.005, num_threads=1, vertical_barrier_times = vertical_barrier_times)\n",
        "bin = labeling.get_bin(events, close)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qMv-bRC74XTt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "945753f0-428e-488a-94ce-1211eea6be65"
      },
      "source": [
        "# bin with side (multi-label)\n",
        "\n",
        "events = labeling.get_events(close, t_events, pt_sl, target, min_ret=0.005, num_threads=1, vertical_barrier_times = vertical_barrier_times ,side_prediction = side)\n",
        "bin = labeling.get_bins(events,close)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/mlfinlab/labeling/labeling.py:124: FutureWarning: \n",
            "Passing list-likes to .loc or [] with any missing label will raise\n",
            "KeyError in the future, you can use .reindex() as an alternative.\n",
            "\n",
            "See the documentation here:\n",
            "https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#deprecate-loc-reindex-listlike\n",
            "  target = target.loc[t_events]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gvb7qObJ6jZ7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "53a8f86a-bf7c-436f-de76-89aaf68acb34"
      },
      "source": [
        "print(bin.bin)\n",
        "# this is label y"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2009-09-30 09:44:47    0\n",
            "2009-09-30 09:56:31    0\n",
            "2009-09-30 12:48:14    0\n",
            "2009-09-30 15:05:38    1\n",
            "2009-10-01 09:52:26    1\n",
            "                      ..\n",
            "2019-06-12 10:18:41    0\n",
            "2019-06-17 09:57:13    1\n",
            "2019-06-18 09:39:43    0\n",
            "2019-06-20 09:30:00    0\n",
            "2019-06-20 12:20:48    1\n",
            "Name: bin, Length: 2665, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yjhEPE-KEOBQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y= bin[['bin']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "971gXX1U9171",
        "colab_type": "text"
      },
      "source": [
        "X (features)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FSSCITo-_1QS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "raw_data = pd.DataFrame()\n",
        "\n",
        "# Log Returns\n",
        "raw_data['log_ret'] = np.log(close).diff()\n",
        "\n",
        "# Momentum\n",
        "raw_data['mom1'] = close.pct_change(periods=1)\n",
        "raw_data['mom2'] = close.pct_change(periods=2)\n",
        "raw_data['mom3'] = close.pct_change(periods=3)\n",
        "raw_data['mom4'] = close.pct_change(periods=4)\n",
        "raw_data['mom5'] = close.pct_change(periods=5)\n",
        "\n",
        "# Volatility\n",
        "raw_data['volatility_50'] = raw_data['log_ret'].rolling(window=50, min_periods=50, center=False).std()\n",
        "raw_data['volatility_31'] = raw_data['log_ret'].rolling(window=31, min_periods=31, center=False).std()\n",
        "raw_data['volatility_15'] = raw_data['log_ret'].rolling(window=15, min_periods=15, center=False).std()\n",
        "\n",
        "# Serial Correlation (Takes about 4 minutes)\n",
        "window_autocorr = 50\n",
        "\n",
        "raw_data['autocorr_1'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=1), raw=False)\n",
        "raw_data['autocorr_2'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=2), raw=False)\n",
        "raw_data['autocorr_3'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=3), raw=False)\n",
        "raw_data['autocorr_4'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=4), raw=False)\n",
        "raw_data['autocorr_5'] = raw_data['log_ret'].rolling(window=window_autocorr, min_periods=window_autocorr, center=False).apply(lambda x: x.autocorr(lag=5), raw=False)\n",
        "\n",
        "# Get the various log -t returns\n",
        "raw_data['log_t1'] = raw_data['log_ret'].shift(1)\n",
        "raw_data['log_t2'] = raw_data['log_ret'].shift(2)\n",
        "raw_data['log_t3'] = raw_data['log_ret'].shift(3)\n",
        "raw_data['log_t4'] = raw_data['log_ret'].shift(4)\n",
        "raw_data['log_t5'] = raw_data['log_ret'].shift(5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hD0xKRhCAa6m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "raw_data = raw_data.shift(1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i9Ud5V5EBfdW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "raw_data['side'] = side"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3qRSIvhW_CmO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "f43db1bf-e9f8-40ba-9d41-691f45601e44"
      },
      "source": [
        "raw_data2 = raw_data.dropna()\n",
        "X = raw_data2.loc[y.index,:]"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:1418: FutureWarning: \n",
            "Passing list-likes to .loc or [] with any missing label will raise\n",
            "KeyError in the future, you can use .reindex() as an alternative.\n",
            "\n",
            "See the documentation here:\n",
            "https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#deprecate-loc-reindex-listlike\n",
            "  return self._getitem_tuple(key)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iaIQBNalEyFr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "outputId": "49280430-622b-4496-97a6-51bdcf353da5"
      },
      "source": [
        "data = X.join(y,how='right').dropna()\n",
        "data.head()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>log_ret</th>\n",
              "      <th>mom1</th>\n",
              "      <th>mom2</th>\n",
              "      <th>mom3</th>\n",
              "      <th>mom4</th>\n",
              "      <th>mom5</th>\n",
              "      <th>volatility_50</th>\n",
              "      <th>volatility_31</th>\n",
              "      <th>volatility_15</th>\n",
              "      <th>autocorr_1</th>\n",
              "      <th>autocorr_2</th>\n",
              "      <th>autocorr_3</th>\n",
              "      <th>autocorr_4</th>\n",
              "      <th>autocorr_5</th>\n",
              "      <th>log_t1</th>\n",
              "      <th>log_t2</th>\n",
              "      <th>log_t3</th>\n",
              "      <th>log_t4</th>\n",
              "      <th>log_t5</th>\n",
              "      <th>side</th>\n",
              "      <th>bin</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2009-10-01 09:52:26</th>\n",
              "      <td>-0.002947</td>\n",
              "      <td>-0.002942</td>\n",
              "      <td>-0.006450</td>\n",
              "      <td>-0.006062</td>\n",
              "      <td>-0.004699</td>\n",
              "      <td>-0.010319</td>\n",
              "      <td>0.002759</td>\n",
              "      <td>0.003416</td>\n",
              "      <td>0.003565</td>\n",
              "      <td>-0.048314</td>\n",
              "      <td>0.173996</td>\n",
              "      <td>-0.255830</td>\n",
              "      <td>0.050401</td>\n",
              "      <td>-0.127175</td>\n",
              "      <td>-0.003525</td>\n",
              "      <td>0.000391</td>\n",
              "      <td>0.001370</td>\n",
              "      <td>-0.005662</td>\n",
              "      <td>-0.000710</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009-10-01 11:16:04</th>\n",
              "      <td>0.002571</td>\n",
              "      <td>0.002575</td>\n",
              "      <td>0.002358</td>\n",
              "      <td>-0.000395</td>\n",
              "      <td>-0.000395</td>\n",
              "      <td>0.000989</td>\n",
              "      <td>0.002866</td>\n",
              "      <td>0.002943</td>\n",
              "      <td>0.002367</td>\n",
              "      <td>-0.075463</td>\n",
              "      <td>0.143150</td>\n",
              "      <td>-0.197550</td>\n",
              "      <td>0.057683</td>\n",
              "      <td>-0.151334</td>\n",
              "      <td>-0.000216</td>\n",
              "      <td>-0.002751</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.001383</td>\n",
              "      <td>-0.004538</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009-10-01 15:51:53</th>\n",
              "      <td>0.000200</td>\n",
              "      <td>0.000200</td>\n",
              "      <td>0.000148</td>\n",
              "      <td>-0.000997</td>\n",
              "      <td>-0.001774</td>\n",
              "      <td>-0.003184</td>\n",
              "      <td>0.002846</td>\n",
              "      <td>0.002476</td>\n",
              "      <td>0.002253</td>\n",
              "      <td>-0.124987</td>\n",
              "      <td>0.026404</td>\n",
              "      <td>-0.106176</td>\n",
              "      <td>0.001387</td>\n",
              "      <td>-0.147859</td>\n",
              "      <td>-0.000052</td>\n",
              "      <td>-0.001145</td>\n",
              "      <td>-0.000778</td>\n",
              "      <td>-0.001414</td>\n",
              "      <td>0.001394</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009-10-01 16:16:36</th>\n",
              "      <td>-0.002206</td>\n",
              "      <td>-0.002203</td>\n",
              "      <td>-0.005390</td>\n",
              "      <td>-0.005192</td>\n",
              "      <td>-0.005243</td>\n",
              "      <td>-0.006382</td>\n",
              "      <td>0.002730</td>\n",
              "      <td>0.002351</td>\n",
              "      <td>0.002189</td>\n",
              "      <td>-0.003966</td>\n",
              "      <td>-0.059657</td>\n",
              "      <td>-0.076941</td>\n",
              "      <td>-0.003204</td>\n",
              "      <td>-0.125404</td>\n",
              "      <td>-0.003199</td>\n",
              "      <td>0.000200</td>\n",
              "      <td>-0.000052</td>\n",
              "      <td>-0.001145</td>\n",
              "      <td>-0.000778</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009-10-02 09:32:42</th>\n",
              "      <td>0.009919</td>\n",
              "      <td>0.009968</td>\n",
              "      <td>0.007743</td>\n",
              "      <td>0.004524</td>\n",
              "      <td>0.004724</td>\n",
              "      <td>0.004672</td>\n",
              "      <td>0.003018</td>\n",
              "      <td>0.002958</td>\n",
              "      <td>0.003157</td>\n",
              "      <td>-0.011765</td>\n",
              "      <td>-0.137563</td>\n",
              "      <td>-0.046403</td>\n",
              "      <td>-0.006264</td>\n",
              "      <td>-0.085668</td>\n",
              "      <td>-0.002206</td>\n",
              "      <td>-0.003199</td>\n",
              "      <td>0.000200</td>\n",
              "      <td>-0.000052</td>\n",
              "      <td>-0.001145</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      log_ret      mom1      mom2  ...    log_t5  side  bin\n",
              "2009-10-01 09:52:26 -0.002947 -0.002942 -0.006450  ... -0.000710  -1.0    1\n",
              "2009-10-01 11:16:04  0.002571  0.002575  0.002358  ... -0.004538  -1.0    1\n",
              "2009-10-01 15:51:53  0.000200  0.000200  0.000148  ...  0.001394  -1.0    1\n",
              "2009-10-01 16:16:36 -0.002206 -0.002203 -0.005390  ... -0.000778  -1.0    1\n",
              "2009-10-02 09:32:42  0.009919  0.009968  0.007743  ... -0.001145  -1.0    0\n",
              "\n",
              "[5 rows x 21 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7NGtd7yu_BKP",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    }
  ]
}